{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import sys\n",
    "import os\n",
    "import random\n",
    "from nltk.stem import *\n",
    "from typing import Set\n",
    "\n",
    "class GetStopWrods():\n",
    "    \n",
    "    def GetListOfStopWords(self, filepath):\n",
    "        stopWordsFile = open(filepath)\n",
    "        stopWordsContext = stopWordsFile.read()\n",
    "        stopWordsList = stopWordsContext.split('\\n')\n",
    "        stopWordsFile.close()\n",
    "        return stopWordsList\n",
    "\n",
    "    def Call(self, filepath):\n",
    "        return self.GetListOfStopWords(filepath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class Cut():\n",
    "\n",
    "    def __init__(self, listOfStopWords = []):\n",
    "        self.regList = [r\".*: .*\\n\", r\">\", r\"[a-zA-Z0-9_-]+@[a-zA-Z0-9_-]+(\\.[a-zA-Z0-9_-]+)+\", r\"\\$\\d\"]\n",
    "        self.replaceItems = [r\"|\", r\"<\", r\">\", r\"(\", r\")\", r\"[\", r\"]\", r\",\", r\".\", r\"?\", r\"!\", r\"\\\"\", r\"-\", r\"_\", r\":\", r\"~\", r\"*\", r\"&\"]\n",
    "        self.listOfStopWords = listOfStopWords\n",
    "\n",
    "\n",
    "    def replaceWithStopWordsAndRegex(self, filename):\n",
    "        f = open(filename)\n",
    "        try:\n",
    "            fContext = f.read()\n",
    "        except UnicodeDecodeError:\n",
    "            f.close()\n",
    "            return \"\"\n",
    "        f.close()\n",
    "        for item in self.regList:\n",
    "            pattern = re.compile(item)\n",
    "            fContext = pattern.sub(\"\", fContext)\n",
    "        for item in self.replaceItems:\n",
    "            fContext = fContext.replace(item, \" \")\n",
    "        return fContext\n",
    "\n",
    "    def cutWords(self, filename):\n",
    "        fContext = self.replaceWithStopWordsAndRegex(filename)\n",
    "        if fContext == \"\":\n",
    "            return []\n",
    "        fContext = fContext.lower()\n",
    "        listOfF = fContext.split()\n",
    "        listOfF = list(set(listOfF))\n",
    "        for item in self.listOfStopWords:\n",
    "            if item in listOfF:\n",
    "                listOfF.remove(item)\n",
    "        #self.wordStemming(listOfF)\n",
    "        return listOfF\n",
    "    \n",
    "    def wordStemming(self, listOfF):\n",
    "        porterStemmer = lancaster.LancasterStemmer()\n",
    "        for i in range(len(listOfF)):\n",
    "            listOfF[i] = porterStemmer.stem(listOfF[i])\n",
    "\n",
    "    def Call(self, filename):\n",
    "        return self.cutWords(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Set\n",
    "import os\n",
    "import random\n",
    "\n",
    "def sampleNFold(totalNumber, N = 5):\n",
    "    container = [i for i in range(totalNumber)]\n",
    "    numberPerFold = int(totalNumber / N)\n",
    "    sampledContainer = []\n",
    "    for i in range(N):\n",
    "        c = random.sample(container, numberPerFold)\n",
    "        sampledContainer.append(c)\n",
    "        for item in c:\n",
    "            container.remove(item)\n",
    "    return sampledContainer\n",
    "\n",
    "class SetProbability():\n",
    "\n",
    "    def __init__(self, N = 5):\n",
    "        self.N = N\n",
    "        self.sampledContainer = sampleNFold(997, self.N)\n",
    "        self.defaults = {}\n",
    "        self.classPaths = []\n",
    "        self.wordsContainer = []\n",
    "        self.wordsDictionary = {}\n",
    "        CLASSES_PATH = os.path.abspath('.') + \"/\" + \"20_newsgroups\" + \"/\"\n",
    "        CLASSES_LIST = os.listdir(CLASSES_PATH)\n",
    "        for dir in CLASSES_LIST:\n",
    "            classPath = CLASSES_PATH + dir + \"/\"\n",
    "            self.classPaths.append(classPath)\n",
    "            files = os.listdir(classPath)\n",
    "            self.__dict__[dir + \"files\"] = files\n",
    "            # print(len(files))\n",
    "            self.defaults[dir + \"WordMatrix\"] = []\n",
    "            self.defaults[dir + \"BoolMatrix\"] = []\n",
    "            self.defaults[dir + \"WordContainer\"] = []\n",
    "            self.defaults[dir + \"WordDictionary\"] = {}\n",
    "        self.__dict__.update(self.defaults)\n",
    "\n",
    "    def clearLastFold(self):\n",
    "        self.defaults = {}\n",
    "        # self.classPaths = []\n",
    "        self.wordsContainer = []\n",
    "        self.wordsDictionary = {}\n",
    "        CLASSES_PATH = os.path.abspath('.') + \"/\" + \"20_newsgroups\" + \"/\"\n",
    "        CLASSES_LIST = os.listdir(CLASSES_PATH)\n",
    "        for dir in CLASSES_LIST:\n",
    "            self.defaults[dir + \"WordMatrix\"] = []\n",
    "            self.defaults[dir + \"BoolMatrix\"] = []\n",
    "            self.defaults[dir + \"WordContainer\"] = []\n",
    "            self.defaults[dir + \"WordDictionary\"] = {}\n",
    "        self.__dict__.update(self.defaults)\n",
    "\n",
    "\n",
    "    def setWordsContainer(self, kth):\n",
    "\n",
    "        CLASSES_PATH = os.path.abspath('.') + \"/\" + \"20_newsgroups\" + \"/\"\n",
    "        CLASSES_LIST = os.listdir(CLASSES_PATH)\n",
    "        \n",
    "        self.clearLastFold()\n",
    "\n",
    "        for dir in CLASSES_LIST:\n",
    "            # print(CLASSES_PATH + dir + \"/\")\n",
    "            classPath = CLASSES_PATH + dir + \"/\"\n",
    "            # self.classPaths.append(classPath)\n",
    "            files = self.__dict__[dir + \"files\"]\n",
    "            flagOfNumber = 0\n",
    "            for file in files:\n",
    "                if flagOfNumber in self.sampledContainer[kth]:\n",
    "                    flagOfNumber += 1\n",
    "                    continue\n",
    "                flagOfNumber += 1\n",
    "                filepath = classPath + file\n",
    "                stopWordsPath = \"stopwords.txt\"\n",
    "                g = GetStopWrods()\n",
    "                stopWords = g.Call(stopWordsPath)\n",
    "                c = Cut(stopWords)\n",
    "                wordsList = c.Call(filepath)\n",
    "                self.__dict__[dir + \"WordMatrix\"].append(wordsList)\n",
    "                self.__dict__[dir + \"WordContainer\"].extend(wordsList)\n",
    "                self.__dict__[dir + \"WordContainer\"] = list(set(self.__dict__[dir + \"WordContainer\"]))\n",
    "                self.wordsContainer.extend(wordsList)\n",
    "                self.wordsContainer = list(set(self.wordsContainer))\n",
    "        print(str(len(self.wordsContainer)) + \"\\n\")\n",
    "\n",
    "    def setWordsDictionary(self):\n",
    "        print(\"WordsDictionary Set begin.\" + \"\\n\")\n",
    "        for i in range(len(self.wordsContainer)):\n",
    "            self.wordsDictionary[self.wordsContainer[i]] = i\n",
    "        CLASSES_PATH = os.path.abspath('.') + \"/\" + \"20_newsgroups\" + \"/\"\n",
    "        CLASSES_LIST = os.listdir(CLASSES_PATH)\n",
    "        for dir in CLASSES_LIST:\n",
    "            for i in range(len(self.__dict__[dir + \"WordContainer\"])):\n",
    "                self.__dict__[dir + \"WordDictionary\"][self.__dict__[dir + \"WordContainer\"][i]] = i\n",
    "        print(\"WordsDictionary Set end.\" + \"\\n\")\n",
    "    \n",
    "    def setBoolMatrix(self):\n",
    "        print(\"BoolMatrix Set begin.\" + \"\\n\")\n",
    "        CLASSES_PATH = os.path.abspath('.') + \"/\" + \"20_newsgroups\" + \"/\"\n",
    "        CLASSES_LIST = os.listdir(CLASSES_PATH)\n",
    "        for dir in CLASSES_LIST:\n",
    "            # print(\"dir \\n\")\n",
    "            for list in self.__dict__[dir + \"WordMatrix\"]:\n",
    "                tmpList = [0 for i in range(len(self.__dict__[dir + \"WordContainer\"]))]\n",
    "                for i in range(len(list)):\n",
    "                    if tmpList[self.__dict__[dir + \"WordDictionary\"][list[i]]] == 0:\n",
    "                        tmpList[self.__dict__[dir + \"WordDictionary\"][list[i]]] = 1\n",
    "                self.__dict__[dir + \"BoolMatrix\"].append(tmpList)\n",
    "        print(\"BoolMatrix Set end.\" + \"\\n\")\n",
    "\n",
    "    def setConditionalProbability(self):\n",
    "        print(\"ConditionalProbability Set begin.\" + \"\\n\")\n",
    "        CLASSES_PATH = os.path.abspath('.') + \"/\" + \"20_newsgroups\" + \"/\"\n",
    "        CLASSES_LIST = os.listdir(CLASSES_PATH)\n",
    "        for dir in CLASSES_LIST:\n",
    "            # print(\"dir \\n\")\n",
    "            SUM = len(self.__dict__[dir + \"WordContainer\"])\n",
    "            self.__dict__[dir + \"CP\"] = [1.0 for i in range(len(self.__dict__[dir + \"WordContainer\"]))]\n",
    "            for list in range(len(self.__dict__[dir + \"BoolMatrix\"])):\n",
    "                for i in range(len(self.__dict__[dir + \"WordContainer\"])):\n",
    "                    self.__dict__[dir + \"CP\"][i] = self.__dict__[dir + \"CP\"][i] + self.__dict__[dir + \"BoolMatrix\"][list][i]\n",
    "                    SUM = SUM + self.__dict__[dir + \"BoolMatrix\"][list][i]\n",
    "            self.__dict__[dir + \"SUM\"] = SUM\n",
    "            for i in range(len(self.__dict__[dir + \"WordContainer\"])):\n",
    "                self.__dict__[dir + \"CP\"][i] = self.__dict__[dir + \"CP\"][i] / SUM\n",
    "        print(\"ConditionalProbability Set end.\" + \"\\n\")\n",
    "    \n",
    "    def train(self):\n",
    "        for i in range(self.N):\n",
    "            self.setWordsContainer(i)\n",
    "            self.setWordsDictionary()\n",
    "            self.setBoolMatrix()\n",
    "            self.setConditionalProbability()\n",
    "            self.verify(i)\n",
    "            predClass, predProbability = self.predict(\"52558\")\n",
    "            print(predClass)\n",
    "\n",
    "    def predict(self, filepath):\n",
    "        print(\"Predict begin.\" + \"\\n\")\n",
    "        stopWordsPath = \"stopwords.txt\"\n",
    "        g = GetStopWrods()\n",
    "        stopWords = g.Call(stopWordsPath)\n",
    "        c = Cut(stopWords)\n",
    "        wordsList = c.Call(filepath)\n",
    "        CLASSES_PATH = os.path.abspath('.') + \"/\" + \"20_newsgroups\" + \"/\"\n",
    "        CLASSES_LIST = os.listdir(CLASSES_PATH)\n",
    "        maxProbability = 0.0\n",
    "        maxClass = None\n",
    "        for dir in CLASSES_LIST:\n",
    "            tmpProbability = 1.0\n",
    "            for i in range(len(wordsList)):\n",
    "                if wordsList[i] in self.__dict__[dir + \"WordDictionary\"]:\n",
    "                    tmpProbability = tmpProbability * self.__dict__[dir + \"CP\"][self.__dict__[dir + \"WordDictionary\"][wordsList[i]]]\n",
    "                else:\n",
    "                    tmpProbability = tmpProbability / (self.__dict__[dir + \"SUM\"] + 1)\n",
    "            if tmpProbability > maxProbability:\n",
    "                maxProbability = tmpProbability\n",
    "                maxClass = dir\n",
    "        print(\"Predict end.\" + \"\\n\")\n",
    "        return maxClass, maxProbability\n",
    "\n",
    "    def verify(self, kth):\n",
    "        print(\"Verify begin.\" + \"\\n\")\n",
    "        T = 0\n",
    "        F = 0\n",
    "        stopWordsPath = \"stopwords.txt\"\n",
    "        CLASSES_PATH = os.path.abspath('.') + \"/\" + \"20_newsgroups\" + \"/\"\n",
    "        CLASSES_LIST = os.listdir(CLASSES_PATH)\n",
    "        for dir in CLASSES_LIST:\n",
    "            classPath = CLASSES_PATH + dir + \"/\"\n",
    "            for item in self.sampledContainer[kth]:\n",
    "                filepath = classPath + self.__dict__[dir + \"files\"][item]\n",
    "                g = GetStopWrods()\n",
    "                stopWords = g.Call(stopWordsPath)\n",
    "                c = Cut(stopWords)\n",
    "                wordsList = c.Call(filepath)\n",
    "\n",
    "                maxProbability = 0.0\n",
    "                maxClass = None\n",
    "                for dir2 in CLASSES_LIST:\n",
    "                    tmpProbability = 1.0\n",
    "                    for i in range(len(wordsList)):\n",
    "                        if wordsList[i] in self.__dict__[dir2 + \"WordDictionary\"]:\n",
    "                            tmpProbability = tmpProbability * self.__dict__[dir2 + \"CP\"][self.__dict__[dir2 + \"WordDictionary\"][wordsList[i]]]\n",
    "                        else:\n",
    "                            tmpProbability = tmpProbability / (self.__dict__[dir2 + \"SUM\"] + 1)\n",
    "                    if tmpProbability > maxProbability:\n",
    "                        maxProbability = tmpProbability\n",
    "                        maxClass = dir2\n",
    "                if maxClass == dir:\n",
    "                    T += 1\n",
    "                else:\n",
    "                    F += 1\n",
    "        print(\"Cross Verify {}: Precisely predicted: {}, Mistakely predicted: {}\".format(kth, T, F))\n",
    "        print(\"Verify end.\" + \"\\n\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "189032\n",
      "\n",
      "WordsDictionary Set begin.\n",
      "\n",
      "WordsDictionary Set end.\n",
      "\n",
      "BoolMatrix Set begin.\n",
      "\n",
      "BoolMatrix Set end.\n",
      "\n",
      "ConditionalProbability Set begin.\n",
      "\n",
      "ConditionalProbability Set end.\n",
      "\n",
      "Verify begin.\n",
      "\n",
      "Cross Verify 0: Precisely predicted: 2223, Mistakely predicted: 1757\n",
      "Verify end.\n",
      "\n",
      "Predict begin.\n",
      "\n",
      "Predict end.\n",
      "\n",
      "rec.sport.hockey\n",
      "189742\n",
      "\n",
      "WordsDictionary Set begin.\n",
      "\n",
      "WordsDictionary Set end.\n",
      "\n",
      "BoolMatrix Set begin.\n",
      "\n",
      "BoolMatrix Set end.\n",
      "\n",
      "ConditionalProbability Set begin.\n",
      "\n",
      "ConditionalProbability Set end.\n",
      "\n",
      "Verify begin.\n",
      "\n",
      "Cross Verify 1: Precisely predicted: 2146, Mistakely predicted: 1834\n",
      "Verify end.\n",
      "\n",
      "Predict begin.\n",
      "\n",
      "Predict end.\n",
      "\n",
      "rec.sport.hockey\n",
      "186847\n",
      "\n",
      "WordsDictionary Set begin.\n",
      "\n",
      "WordsDictionary Set end.\n",
      "\n",
      "BoolMatrix Set begin.\n",
      "\n",
      "BoolMatrix Set end.\n",
      "\n",
      "ConditionalProbability Set begin.\n",
      "\n",
      "ConditionalProbability Set end.\n",
      "\n",
      "Verify begin.\n",
      "\n",
      "Cross Verify 2: Precisely predicted: 2216, Mistakely predicted: 1764\n",
      "Verify end.\n",
      "\n",
      "Predict begin.\n",
      "\n",
      "Predict end.\n",
      "\n",
      "rec.sport.hockey\n",
      "184781\n",
      "\n",
      "WordsDictionary Set begin.\n",
      "\n",
      "WordsDictionary Set end.\n",
      "\n",
      "BoolMatrix Set begin.\n",
      "\n",
      "BoolMatrix Set end.\n",
      "\n",
      "ConditionalProbability Set begin.\n",
      "\n",
      "ConditionalProbability Set end.\n",
      "\n",
      "Verify begin.\n",
      "\n",
      "Cross Verify 3: Precisely predicted: 2200, Mistakely predicted: 1780\n",
      "Verify end.\n",
      "\n",
      "Predict begin.\n",
      "\n",
      "Predict end.\n",
      "\n",
      "None\n",
      "177882\n",
      "\n",
      "WordsDictionary Set begin.\n",
      "\n",
      "WordsDictionary Set end.\n",
      "\n",
      "BoolMatrix Set begin.\n",
      "\n",
      "BoolMatrix Set end.\n",
      "\n",
      "ConditionalProbability Set begin.\n",
      "\n",
      "ConditionalProbability Set end.\n",
      "\n",
      "Verify begin.\n",
      "\n",
      "Cross Verify 4: Precisely predicted: 2118, Mistakely predicted: 1862\n",
      "Verify end.\n",
      "\n",
      "Predict begin.\n",
      "\n",
      "Predict end.\n",
      "\n",
      "None\n",
      "Predict begin.\n",
      "\n",
      "Predict end.\n",
      "\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "s = SetProbability()\n",
    "s.train()\n",
    "predClass, predProbability = s.predict(\"52558\")\n",
    "print(predClass)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/kled/Experiment2\n"
     ]
    }
   ],
   "source": [
    "print(os.path.abspath('.'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164054\n",
      "\n",
      "WordsDictionary Set begin.\n",
      "\n",
      "WordsDictionary Set end.\n",
      "\n",
      "BoolMatrix Set begin.\n",
      "\n",
      "BoolMatrix Set end.\n",
      "\n",
      "ConditionalProbability Set begin.\n",
      "\n",
      "ConditionalProbability Set end.\n",
      "\n",
      "Verify begin.\n",
      "\n",
      "Cross Verify 0: Precisely predicted: 3108, Mistakely predicted: 872\n",
      "F1 Score: 0.7927925041567642.\n",
      "Verify end.\n",
      "\n",
      "Predict begin.\n",
      "\n",
      "Predict end.\n",
      "\n",
      "rec.sport.hockey\n",
      "180584\n",
      "\n",
      "WordsDictionary Set begin.\n",
      "\n",
      "WordsDictionary Set end.\n",
      "\n",
      "BoolMatrix Set begin.\n",
      "\n",
      "BoolMatrix Set end.\n",
      "\n",
      "ConditionalProbability Set begin.\n",
      "\n",
      "ConditionalProbability Set end.\n",
      "\n",
      "Verify begin.\n",
      "\n",
      "Cross Verify 1: Precisely predicted: 3177, Mistakely predicted: 803\n",
      "F1 Score: 0.807409335522783.\n",
      "Verify end.\n",
      "\n",
      "Predict begin.\n",
      "\n",
      "Predict end.\n",
      "\n",
      "rec.sport.hockey\n",
      "190688\n",
      "\n",
      "WordsDictionary Set begin.\n",
      "\n",
      "WordsDictionary Set end.\n",
      "\n",
      "BoolMatrix Set begin.\n",
      "\n",
      "BoolMatrix Set end.\n",
      "\n",
      "ConditionalProbability Set begin.\n",
      "\n",
      "ConditionalProbability Set end.\n",
      "\n",
      "Verify begin.\n",
      "\n",
      "Cross Verify 2: Precisely predicted: 3108, Mistakely predicted: 872\n",
      "F1 Score: 0.7907106073780438.\n",
      "Verify end.\n",
      "\n",
      "Predict begin.\n",
      "\n",
      "Predict end.\n",
      "\n",
      "rec.sport.hockey\n",
      "196095\n",
      "\n",
      "WordsDictionary Set begin.\n",
      "\n",
      "WordsDictionary Set end.\n",
      "\n",
      "BoolMatrix Set begin.\n",
      "\n",
      "BoolMatrix Set end.\n",
      "\n",
      "ConditionalProbability Set begin.\n",
      "\n",
      "ConditionalProbability Set end.\n",
      "\n",
      "Verify begin.\n",
      "\n",
      "Cross Verify 3: Precisely predicted: 3145, Mistakely predicted: 835\n",
      "F1 Score: 0.7969195695395207.\n",
      "Verify end.\n",
      "\n",
      "Predict begin.\n",
      "\n",
      "Predict end.\n",
      "\n",
      "rec.sport.hockey\n",
      "192524\n",
      "\n",
      "WordsDictionary Set begin.\n",
      "\n",
      "WordsDictionary Set end.\n",
      "\n",
      "BoolMatrix Set begin.\n",
      "\n",
      "BoolMatrix Set end.\n",
      "\n",
      "ConditionalProbability Set begin.\n",
      "\n",
      "ConditionalProbability Set end.\n",
      "\n",
      "Verify begin.\n",
      "\n",
      "Cross Verify 4: Precisely predicted: 3104, Mistakely predicted: 876\n",
      "F1 Score: 0.7886486394710267.\n",
      "Verify end.\n",
      "\n",
      "Predict begin.\n",
      "\n",
      "Predict end.\n",
      "\n",
      "rec.sport.hockey\n",
      "Predict begin.\n",
      "\n",
      "Predict end.\n",
      "\n",
      "rec.sport.hockey\n"
     ]
    }
   ],
   "source": [
    "from typing import Set\n",
    "import os\n",
    "import random\n",
    "import pandas as pd\n",
    "\n",
    "def sampleNFold(totalNumber, N = 5):\n",
    "    container = [i for i in range(totalNumber)]\n",
    "    numberPerFold = int(totalNumber / N)\n",
    "    sampledContainer = []\n",
    "    for i in range(N):\n",
    "        c = random.sample(container, numberPerFold)\n",
    "        sampledContainer.append(c)\n",
    "        for item in c:\n",
    "            container.remove(item)\n",
    "    return sampledContainer\n",
    "\n",
    "class SetProbability():\n",
    "\n",
    "    def __init__(self, N = 5):\n",
    "        self.N = N\n",
    "        self.sampledContainer = sampleNFold(997, self.N)\n",
    "        self.defaults = {}\n",
    "        self.classPaths = []\n",
    "        self.wordsContainer = []\n",
    "        self.wordsDictionary = {}\n",
    "        CLASSES_PATH = os.path.abspath('.') + \"/\" + \"20_newsgroups\" + \"/\"\n",
    "        CLASSES_LIST = os.listdir(CLASSES_PATH)\n",
    "        data = [[0 for i in range(len(CLASSES_LIST))] for j in range(len(CLASSES_LIST))]\n",
    "        self.confusionMatrix = pd.DataFrame(data, index = CLASSES_LIST, columns = CLASSES_LIST) \n",
    "        for dir in CLASSES_LIST:\n",
    "            classPath = CLASSES_PATH + dir + \"/\"\n",
    "            self.classPaths.append(classPath)\n",
    "            files = os.listdir(classPath)\n",
    "            self.__dict__[dir + \"files\"] = files\n",
    "            # print(len(files))\n",
    "            self.defaults[dir + \"WordMatrix\"] = []\n",
    "            self.defaults[dir + \"BoolMatrix\"] = []\n",
    "            self.defaults[dir + \"WordContainer\"] = []\n",
    "            self.defaults[dir + \"WordDictionary\"] = {}\n",
    "        self.__dict__.update(self.defaults)\n",
    "\n",
    "        \n",
    "    def computeF1Score(self):\n",
    "        P = 0.0\n",
    "        R = 0.0\n",
    "        for i in range(len(self.confusionMatrix.values)):\n",
    "            tmpR = 0.0\n",
    "            for j in range(len(self.confusionMatrix.values)):\n",
    "                tmpR = tmpR + self.confusionMatrix.values[i][j]\n",
    "            tmpR = self.confusionMatrix.values[i][i] / tmpR\n",
    "            R += tmpR\n",
    "        for i in range(len(self.confusionMatrix.values)):\n",
    "            tmpP = 0.0\n",
    "            for j in range(len(self.confusionMatrix.values)):\n",
    "                tmpP = tmpP + self.confusionMatrix.values[j][i]\n",
    "            tmpP = self.confusionMatrix.values[i][i] / tmpP\n",
    "            P += tmpP\n",
    "        P = P / len(self.confusionMatrix.values)\n",
    "        R = R / len(self.confusionMatrix.values)\n",
    "        return 2 * P * R / (P + R)\n",
    "    \n",
    "    def clearLastFold(self):\n",
    "        self.defaults = {}\n",
    "        # self.classPaths = []\n",
    "        self.wordsContainer = []\n",
    "        self.wordsDictionary = {}\n",
    "        CLASSES_PATH = os.path.abspath('.') + \"/\" + \"20_newsgroups\" + \"/\"\n",
    "        CLASSES_LIST = os.listdir(CLASSES_PATH)\n",
    "        data = [[0 for i in range(len(CLASSES_LIST))] for j in range(len(CLASSES_LIST))]\n",
    "        self.confusionMatrix = pd.DataFrame(data, index = CLASSES_LIST, columns = CLASSES_LIST) \n",
    "        for dir in CLASSES_LIST:\n",
    "            self.defaults[dir + \"WordMatrix\"] = []\n",
    "            self.defaults[dir + \"BoolMatrix\"] = []\n",
    "            self.defaults[dir + \"WordContainer\"] = []\n",
    "            self.defaults[dir + \"WordDictionary\"] = {}\n",
    "        self.__dict__.update(self.defaults)\n",
    "\n",
    "\n",
    "    def setWordsContainer(self, kth):\n",
    "\n",
    "        CLASSES_PATH = os.path.abspath('.') + \"/\" + \"20_newsgroups\" + \"/\"\n",
    "        CLASSES_LIST = os.listdir(CLASSES_PATH)\n",
    "        \n",
    "        self.clearLastFold()\n",
    "\n",
    "        for dir in CLASSES_LIST:\n",
    "            # print(CLASSES_PATH + dir + \"/\")\n",
    "            classPath = CLASSES_PATH + dir + \"/\"\n",
    "            # self.classPaths.append(classPath)\n",
    "            files = self.__dict__[dir + \"files\"]\n",
    "            flagOfNumber = 0\n",
    "            for file in files:\n",
    "                if flagOfNumber in self.sampledContainer[kth]:\n",
    "                    flagOfNumber += 1\n",
    "                    continue\n",
    "                flagOfNumber += 1\n",
    "                filepath = classPath + file\n",
    "                stopWordsPath = \"stopwords.txt\"\n",
    "                g = GetStopWrods()\n",
    "                stopWords = g.Call(stopWordsPath)\n",
    "                c = Cut(stopWords)\n",
    "                wordsList = c.Call(filepath)\n",
    "                self.__dict__[dir + \"WordMatrix\"].append(wordsList)\n",
    "                self.__dict__[dir + \"WordContainer\"].extend(wordsList)\n",
    "                self.__dict__[dir + \"WordContainer\"] = list(set(self.__dict__[dir + \"WordContainer\"]))\n",
    "                self.wordsContainer.extend(wordsList)\n",
    "                self.wordsContainer = list(set(self.wordsContainer))\n",
    "        print(str(len(self.wordsContainer)) + \"\\n\")\n",
    "\n",
    "    def setWordsDictionary(self):\n",
    "        print(\"WordsDictionary Set begin.\" + \"\\n\")\n",
    "        for i in range(len(self.wordsContainer)):\n",
    "            self.wordsDictionary[self.wordsContainer[i]] = i\n",
    "        CLASSES_PATH = os.path.abspath('.') + \"/\" + \"20_newsgroups\" + \"/\"\n",
    "        CLASSES_LIST = os.listdir(CLASSES_PATH)\n",
    "        for dir in CLASSES_LIST:\n",
    "            for i in range(len(self.__dict__[dir + \"WordContainer\"])):\n",
    "                self.__dict__[dir + \"WordDictionary\"][self.__dict__[dir + \"WordContainer\"][i]] = i\n",
    "        print(\"WordsDictionary Set end.\" + \"\\n\")\n",
    "    \n",
    "    def setBoolMatrix(self):\n",
    "        print(\"BoolMatrix Set begin.\" + \"\\n\")\n",
    "        CLASSES_PATH = os.path.abspath('.') + \"/\" + \"20_newsgroups\" + \"/\"\n",
    "        CLASSES_LIST = os.listdir(CLASSES_PATH)\n",
    "        for dir in CLASSES_LIST:\n",
    "            # print(\"dir \\n\")\n",
    "            for list in self.__dict__[dir + \"WordMatrix\"]:\n",
    "                tmpList = [0 for i in range(len(self.__dict__[dir + \"WordContainer\"]))]\n",
    "                for i in range(len(list)):\n",
    "                    if tmpList[self.__dict__[dir + \"WordDictionary\"][list[i]]] == 0:\n",
    "                        tmpList[self.__dict__[dir + \"WordDictionary\"][list[i]]] = 1\n",
    "                self.__dict__[dir + \"BoolMatrix\"].append(tmpList)\n",
    "        print(\"BoolMatrix Set end.\" + \"\\n\")\n",
    "\n",
    "    def setConditionalProbability(self):\n",
    "        print(\"ConditionalProbability Set begin.\" + \"\\n\")\n",
    "        CLASSES_PATH = os.path.abspath('.') + \"/\" + \"20_newsgroups\" + \"/\"\n",
    "        CLASSES_LIST = os.listdir(CLASSES_PATH)\n",
    "        for dir in CLASSES_LIST:\n",
    "            # print(\"dir \\n\")\n",
    "            SUM = len(self.__dict__[dir + \"WordContainer\"])\n",
    "            self.__dict__[dir + \"CP\"] = [1.0 for i in range(len(self.__dict__[dir + \"WordContainer\"]))]\n",
    "            for list in range(len(self.__dict__[dir + \"BoolMatrix\"])):\n",
    "                for i in range(len(self.__dict__[dir + \"WordContainer\"])):\n",
    "                    self.__dict__[dir + \"CP\"][i] = self.__dict__[dir + \"CP\"][i] + self.__dict__[dir + \"BoolMatrix\"][list][i]\n",
    "                    SUM = SUM + self.__dict__[dir + \"BoolMatrix\"][list][i]\n",
    "            self.__dict__[dir + \"SUM\"] = SUM\n",
    "            for i in range(len(self.__dict__[dir + \"WordContainer\"])):\n",
    "                self.__dict__[dir + \"CP\"][i] = self.__dict__[dir + \"CP\"][i] / SUM\n",
    "        print(\"ConditionalProbability Set end.\" + \"\\n\")\n",
    "    \n",
    "    def train(self):\n",
    "        for i in range(self.N):\n",
    "            self.setWordsContainer(i)\n",
    "            self.setWordsDictionary()\n",
    "            self.setBoolMatrix()\n",
    "            self.setConditionalProbability()\n",
    "            self.verify(i)\n",
    "            predClass, predProbability = self.predict(\"52558\")\n",
    "            print(predClass)\n",
    "\n",
    "    def predict(self, filepath):\n",
    "        print(\"Predict begin.\" + \"\\n\")\n",
    "        stopWordsPath = \"stopwords.txt\"\n",
    "        g = GetStopWrods()\n",
    "        stopWords = g.Call(stopWordsPath)\n",
    "        c = Cut(stopWords)\n",
    "        wordsList = c.Call(filepath)\n",
    "        CLASSES_PATH = os.path.abspath('.') + \"/\" + \"20_newsgroups\" + \"/\"\n",
    "        CLASSES_LIST = os.listdir(CLASSES_PATH)\n",
    "        maxProbability = 0.0\n",
    "        maxClass = None\n",
    "        minUnseen = 1000000\n",
    "        for dir in CLASSES_LIST:\n",
    "            tmpProbability = 1.0\n",
    "            unseen = 0\n",
    "            for i in range(len(wordsList)):\n",
    "                if wordsList[i] in self.__dict__[dir + \"WordDictionary\"]:\n",
    "                    tmpProbability = tmpProbability * self.__dict__[dir + \"CP\"][self.__dict__[dir + \"WordDictionary\"][wordsList[i]]]\n",
    "                else:\n",
    "                    unseen += 1\n",
    "                    tmpProbability = tmpProbability / (self.__dict__[dir + \"SUM\"] + 1)\n",
    "            if tmpProbability > maxProbability:\n",
    "                maxProbability = tmpProbability\n",
    "                maxClass = dir\n",
    "            if maxProbability == 0:\n",
    "                if unseen < minUnseen:\n",
    "                    maxClass = dir\n",
    "                    minUnseen = unseen\n",
    "        print(\"Predict end.\" + \"\\n\")\n",
    "        return maxClass, maxProbability\n",
    "\n",
    "    def verify(self, kth):\n",
    "        print(\"Verify begin.\" + \"\\n\")\n",
    "        T = 0\n",
    "        F = 0\n",
    "        stopWordsPath = \"stopwords.txt\"\n",
    "        CLASSES_PATH = os.path.abspath('.') + \"/\" + \"20_newsgroups\" + \"/\"\n",
    "        CLASSES_LIST = os.listdir(CLASSES_PATH)\n",
    "        for dir in CLASSES_LIST:\n",
    "            classPath = CLASSES_PATH + dir + \"/\"\n",
    "            for item in self.sampledContainer[kth]:\n",
    "                filepath = classPath + self.__dict__[dir + \"files\"][item]\n",
    "                g = GetStopWrods()\n",
    "                stopWords = g.Call(stopWordsPath)\n",
    "                c = Cut(stopWords)\n",
    "                wordsList = c.Call(filepath)\n",
    "\n",
    "                maxProbability = 0.0\n",
    "                maxClass = None\n",
    "                minUnseen = 10000000\n",
    "                for dir2 in CLASSES_LIST:\n",
    "                    tmpProbability = 1.0\n",
    "                    unseen = 0\n",
    "                    for i in range(len(wordsList)):\n",
    "                        if wordsList[i] in self.__dict__[dir2 + \"WordDictionary\"]:\n",
    "                            tmpProbability = tmpProbability * self.__dict__[dir2 + \"CP\"][self.__dict__[dir2 + \"WordDictionary\"][wordsList[i]]]\n",
    "                        else:\n",
    "                            unseen += 1\n",
    "                            tmpProbability = tmpProbability / (self.__dict__[dir2 + \"SUM\"] + 1)\n",
    "                    if tmpProbability > maxProbability:\n",
    "                        maxProbability = tmpProbability\n",
    "                        maxClass = dir2\n",
    "                    if maxProbability == 0:\n",
    "                        if unseen < minUnseen:\n",
    "                            maxClass = dir2\n",
    "                            minUnseen = unseen\n",
    "                if maxClass == dir:\n",
    "                    T += 1\n",
    "                else:\n",
    "                    F += 1\n",
    "                self.confusionMatrix.at[dir, maxClass] += 1\n",
    "        print(\"Cross Verify {}: Precisely predicted: {}, Mistakely predicted: {}\".format(kth, T, F))\n",
    "        # print(self.confusionMatrix)\n",
    "        F1 = self.computeF1Score()\n",
    "        print(\"F1 Score: {}.\".format(F1))\n",
    "        csvName = \"confusionMatrix\" + str(kth) + \".csv\"\n",
    "        self.confusionMatrix.to_csv(csvName)\n",
    "        print(\"Verify end.\" + \"\\n\")\n",
    "\n",
    "s = SetProbability()\n",
    "s.train()\n",
    "predClass, predProbability = s.predict(\"52558\")\n",
    "print(predClass)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
